import re

accepted_chars = "a-zA-Z0-9_:"
parts = ["^",                                               # match the beginning of the line
        "[^%(accepted)s]*",                                 # skip over any character that is *NOT* accepted
        "(",                                                # we now match in 2 cases
        "([%(accepted)s]+[^%(accepted)s]+[%(accepted)s]+)", # case 1: we match 1 or more accepted characters,
                                                            # followed by 1 or more a punctuation
                                                            # character, followed by 1 or more accepted characters
        "|",                                                # OR
        "([%(accepted)s]+)",                                # case 2: we match 1 or more accepted characters
        ")", 
        "[^(accepted)s]*$",]                                # match 0 or more unaccepted characters to the end of line
expression = ''.join(parts) % {'accepted': accepted_chars}
PUNC_RE = re.compile(expression)

class Tokenizer(object):
    # Punctuation regex to strip any non-alphanumeric characters from the 
    # front and end

    # We allow words with 1 or more punctuation marks inside the word to allow for 
    # contractions.  The PUNC_RE expression is applied to each white-space delimited
    # word to extract a term

    def __init__(self, data):
        if data is None:
            self._data = []
        elif ' ' in data:
            self._data = [t.lower().strip() for t in data.split()]
        else:
            self._data = [data.lower().strip()]

    def next(self):
        # Loop until we rn out of terms or we return a token
        while self._data:
            term = self._data[0]
            self._data = self._data[1:]
            match = PUNC_RE.match(term)
            if match:
                groups = match.groups()
                if groups:
                    return groups[0]
        raise StopIteration
        
    def __iter__(self):
        return self



